{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型结构\n",
    "\n",
    "- 两层单向LSTM, 输出序列结果, 即(batch_size, step_size, feature_size)\n",
    "- 分别输入到1, 2, 3, 4, 5, 6共6个不同长度的卷积层中\n",
    "- 卷积层为双层, 最后的池化层有Average和Max两种\n",
    "- 对于每个问题, 将所有卷积核结果并起来\n",
    "- 将两个问题并起来的结果, 分别[相减并取绝对值], [x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import os\n",
    "import shutil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    os.mkdir(\"./log/\")\n",
    "    os.mkdir(\"./result/\")\n",
    "except FileExistsError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((10001, 300), (3049, 300))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_PATH = \"./data/\"\n",
    "TRAIN_PATH = DATA_PATH + \"train.csv\"\n",
    "TEST_PATH = DATA_PATH + \"test.csv\"\n",
    "WORD_EMBED_PATH = DATA_PATH + \"word_embed.txt\"\n",
    "CHAR_EMBED_PATH = DATA_PATH + \"char_embed.txt\"\n",
    "QUEST_PATH = DATA_PATH + \"question.csv\"\n",
    "\n",
    "train_data = pd.read_csv(TRAIN_PATH)\n",
    "test_data = pd.read_csv(TEST_PATH)\n",
    "question_data = pd.read_csv(QUEST_PATH)\n",
    "word_embedding_data = pd.read_csv(WORD_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "char_embedding_data = pd.read_csv(CHAR_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "\n",
    "question_data[\"words\"] = question_data[\"words\"].str.split(\" \")\n",
    "question_data[\"chars\"] = question_data[\"chars\"].str.split(\" \")\n",
    "\n",
    "label = train_data[\"label\"].values\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "MAX_COUNT = 10000\n",
    "\n",
    "word_tokenizer = Tokenizer(MAX_COUNT)\n",
    "word_tokenizer.fit_on_texts(question_data[\"words\"])\n",
    "\n",
    "word_embedding_data = np.concatenate(\n",
    "    (\n",
    "        np.zeros(shape=(1, word_embedding_data.shape[1]), dtype=np.float64),\n",
    "        word_embedding_data.loc[list(word_tokenizer.word_index.keys())[:MAX_COUNT]].values\n",
    "    ),\n",
    "    axis=0\n",
    ")\n",
    "\n",
    "char_tokenizer = Tokenizer(MAX_COUNT)\n",
    "char_tokenizer.fit_on_texts(question_data[\"chars\"])\n",
    "\n",
    "char_embedding_data = np.concatenate(\n",
    "    (\n",
    "        np.zeros(shape=(1, char_embedding_data.shape[1]), dtype=np.float64),\n",
    "        char_embedding_data.loc[list(char_tokenizer.word_index.keys())[:MAX_COUNT]].values\n",
    "    ),\n",
    "    axis=0\n",
    ")\n",
    "\n",
    "word_embedding_data.shape, char_embedding_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((254386, 25),\n",
       " (254386, 25),\n",
       " (172956, 25),\n",
       " (172956, 25),\n",
       " (254386, 25),\n",
       " (254386, 25),\n",
       " (172956, 25),\n",
       " (172956, 25))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "SEQ_LEN = 25\n",
    "\n",
    "def gen_word_data(data):\n",
    "    seq_word1 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q1\", right_on=\"qid\")[\"words\"])\n",
    "    seq_word2 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q2\", right_on=\"qid\")[\"words\"])\n",
    "    return pad_sequences(seq_word1, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\"), \\\n",
    "        pad_sequences(seq_word2, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\")\n",
    "    \n",
    "def gen_char_data(data):\n",
    "    seq_char1 = char_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q1\", right_on=\"qid\")[\"chars\"])\n",
    "    seq_char2 = char_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q2\", right_on=\"qid\")[\"chars\"])\n",
    "    return pad_sequences(seq_char1, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\"), \\\n",
    "        pad_sequences(seq_char2, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\")\n",
    "\n",
    "word1, word2 = gen_word_data(train_data)\n",
    "char1, char2 = gen_char_data(train_data)\n",
    "test_word1, test_word2 = gen_word_data(test_data)\n",
    "test_char1, test_char2 = gen_char_data(test_data)\n",
    "\n",
    "word1.shape, word2.shape, test_word1.shape, test_word2.shape, char1.shape, char2.shape, test_char1.shape, test_char2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.optimizers import Adam, SGD, Nadam\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras.layers import LSTM, Bidirectional, TimeDistributed\n",
    "from keras.layers import Conv1D, MaxPool1D, GlobalAveragePooling1D\n",
    "from keras.layers import Input, Embedding, Dropout, BatchNormalization, Dense, Flatten, Lambda, K, Activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# general\n",
    "NUM_EPOCHES = 50\n",
    "EPOCHES1 = 5\n",
    "EPOCHES2 = 25 # 5\n",
    "EPOCHES3 = 22\n",
    "BATCH_SIZE = 1024\n",
    "DROP_RATE = 0.3\n",
    "\n",
    "# cnn\n",
    "CONV_LEN1 = 128\n",
    "CONV_LEN2 = 128\n",
    "CONV_LEN3 = 128\n",
    "CONV_LEN4 = 128\n",
    "CONV_LEN5 = 128\n",
    "CONV_LEN6 = 128\n",
    "\n",
    "# lstm\n",
    "LSTM_SIZE1 = 256\n",
    "LSTM_SIZE2 = 256\n",
    "LSTM_DROP_RATE = 0.3\n",
    "\n",
    "# dense\n",
    "DENSE_SIZE1 = 512\n",
    "DENSE_SIZE2 = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cnn_layer1(inputa, inputb, filters, kernel_size): # with average pooling\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa = GlobalAveragePooling1D()(conv_outputa)\n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb = GlobalAveragePooling1D()(conv_outputb)\n",
    "    return conv_outputa, conv_outputb\n",
    "    \n",
    "def cnn_layer2(inputa, inputb, filters, kernel_size): # with max pooling\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa = MaxPool1D(pool_size=SEQ_LEN)(conv_outputa)\n",
    "    conv_outputa = Flatten()(conv_outputa)\n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb = MaxPool1D(pool_size=SEQ_LEN)(conv_outputb)\n",
    "    conv_outputb = Flatten()(conv_outputb)\n",
    "    return conv_outputa, conv_outputb\n",
    "\n",
    "def cnn_layer3(inputa, inputb, filters, kernel_size): # with both max and average poolings\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    \n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa1 = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv_outputa))\n",
    "    conv_outputa2 = GlobalAveragePooling1D()(conv_outputa)\n",
    "    conv_outputa = concatenate([conv_outputa1, conv_outputa2])\n",
    "    \n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb1 = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv_outputb))\n",
    "    conv_outputb2 = GlobalAveragePooling1D()(conv_outputb)\n",
    "    conv_outputb = concatenate([conv_outputb1, conv_outputb2])\n",
    "    \n",
    "    return conv_outputa, conv_outputb\n",
    "\n",
    "def cnn_double_layer(inputa, inputb, filters, kernel_size):\n",
    "    conv1 = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\")\n",
    "    conv2 = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\")\n",
    "    \n",
    "    conv1a = conv1(inputa)\n",
    "    conv1a = BatchNormalization()(conv1a)\n",
    "    conv1a = Activation(activation=\"relu\")(conv1a)\n",
    "    conv2a = conv2(conv1a)\n",
    "    conv2a = BatchNormalization()(conv2a)\n",
    "    conv2a = Activation(activation=\"relu\")(conv2a)\n",
    "    output_avg_a = GlobalAveragePooling1D()(conv2a)\n",
    "    output_max_a = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv2a))\n",
    "    output_a = concatenate([output_avg_a, output_max_a])\n",
    "    \n",
    "    conv1b = conv1(inputb)\n",
    "    conv1b = BatchNormalization()(conv1b)\n",
    "    conv1b = Activation(activation=\"relu\")(conv1b)\n",
    "    conv2b = conv2(conv1b)\n",
    "    conv2b = BatchNormalization()(conv2b)\n",
    "    conv2b = Activation(activation=\"relu\")(conv2b)\n",
    "    output_avg_b = GlobalAveragePooling1D()(conv2b)\n",
    "    output_max_b = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv2b))\n",
    "    output_b = concatenate([output_avg_b, output_max_b])\n",
    "    \n",
    "    return output_a, output_b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py:497: calling conv1d (from tensorflow.python.ops.nn_ops) with data_format=NHWC is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "`NHWC` for data_format is deprecated, use `NWC` instead\n",
      "Train on 228946 samples, validate on 25440 samples\n",
      "Epoch 1/50\n",
      "228946/228946 [==============================] - 197s 863us/step - loss: 0.4540 - acc: 0.7869 - val_loss: 0.3480 - val_acc: 0.8550\n",
      "Epoch 2/50\n",
      "228946/228946 [==============================] - 184s 804us/step - loss: 0.3115 - acc: 0.8632 - val_loss: 0.2688 - val_acc: 0.8847\n",
      "Epoch 3/50\n",
      "228946/228946 [==============================] - 187s 818us/step - loss: 0.2691 - acc: 0.8842 - val_loss: 0.2407 - val_acc: 0.8981\n",
      "Epoch 4/50\n",
      "228946/228946 [==============================] - 194s 848us/step - loss: 0.2429 - acc: 0.8963 - val_loss: 0.2196 - val_acc: 0.9096\n",
      "Epoch 5/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.2249 - acc: 0.9048 - val_loss: 0.2227 - val_acc: 0.9097\n",
      "Epoch 6/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.2112 - acc: 0.9111 - val_loss: 0.2120 - val_acc: 0.9131\n",
      "Epoch 7/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.2008 - acc: 0.9162 - val_loss: 0.2037 - val_acc: 0.9193\n",
      "Epoch 8/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1926 - acc: 0.9198 - val_loss: 0.1998 - val_acc: 0.9221\n",
      "Epoch 9/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1857 - acc: 0.9220 - val_loss: 0.1887 - val_acc: 0.9238\n",
      "Epoch 10/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1803 - acc: 0.9249 - val_loss: 0.1924 - val_acc: 0.9241\n",
      "Epoch 11/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1740 - acc: 0.9276 - val_loss: 0.1883 - val_acc: 0.9233\n",
      "Epoch 12/50\n",
      "228946/228946 [==============================] - 182s 797us/step - loss: 0.1686 - acc: 0.9306 - val_loss: 0.1891 - val_acc: 0.9264\n",
      "Epoch 13/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1663 - acc: 0.9316 - val_loss: 0.1942 - val_acc: 0.9261\n",
      "Epoch 14/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.1621 - acc: 0.9329 - val_loss: 0.1881 - val_acc: 0.9259\n",
      "Epoch 15/50\n",
      "228946/228946 [==============================] - 182s 797us/step - loss: 0.1581 - acc: 0.9346 - val_loss: 0.1864 - val_acc: 0.9268\n",
      "Epoch 16/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.1554 - acc: 0.9357 - val_loss: 0.1953 - val_acc: 0.9279\n",
      "Epoch 17/50\n",
      "228946/228946 [==============================] - 184s 802us/step - loss: 0.1530 - acc: 0.9367 - val_loss: 0.1891 - val_acc: 0.9291\n",
      "Epoch 18/50\n",
      "228946/228946 [==============================] - 191s 834us/step - loss: 0.1484 - acc: 0.9386 - val_loss: 0.1930 - val_acc: 0.9273\n",
      "Epoch 19/50\n",
      "228946/228946 [==============================] - 196s 854us/step - loss: 0.1464 - acc: 0.9398 - val_loss: 0.1819 - val_acc: 0.9295\n",
      "Epoch 20/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1446 - acc: 0.9403 - val_loss: 0.1821 - val_acc: 0.9302\n",
      "Epoch 21/50\n",
      "228946/228946 [==============================] - 183s 799us/step - loss: 0.1415 - acc: 0.9412 - val_loss: 0.1891 - val_acc: 0.9293\n",
      "Epoch 22/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1398 - acc: 0.9422 - val_loss: 0.1881 - val_acc: 0.9260\n",
      "Epoch 23/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1381 - acc: 0.9432 - val_loss: 0.1896 - val_acc: 0.9284\n",
      "Epoch 24/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1277 - acc: 0.9476 - val_loss: 0.1856 - val_acc: 0.9317\n",
      "Epoch 25/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1205 - acc: 0.9503 - val_loss: 0.1902 - val_acc: 0.9322\n",
      "Epoch 26/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1171 - acc: 0.9522 - val_loss: 0.1840 - val_acc: 0.9323\n",
      "Epoch 27/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1138 - acc: 0.9534 - val_loss: 0.1905 - val_acc: 0.9307\n",
      "load model ./log/20180714-010207.Multi_LSTM_CNN_v3.word.019.hdf5\n",
      "Train on 228946 samples, validate on 25440 samples\n",
      "Epoch 1/50\n",
      "228946/228946 [==============================] - 190s 828us/step - loss: 0.4576 - acc: 0.7841 - val_loss: 0.3596 - val_acc: 0.8581\n",
      "Epoch 2/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.3137 - acc: 0.8608 - val_loss: 0.2810 - val_acc: 0.8854\n",
      "Epoch 3/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.2708 - acc: 0.8828 - val_loss: 0.2509 - val_acc: 0.8967\n",
      "Epoch 4/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.2440 - acc: 0.8965 - val_loss: 0.2204 - val_acc: 0.9081\n",
      "Epoch 5/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.2263 - acc: 0.9036 - val_loss: 0.2152 - val_acc: 0.9107\n",
      "Epoch 6/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.2119 - acc: 0.9110 - val_loss: 0.2024 - val_acc: 0.9184\n",
      "Epoch 7/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.2018 - acc: 0.9150 - val_loss: 0.1989 - val_acc: 0.9221\n",
      "Epoch 8/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1942 - acc: 0.9191 - val_loss: 0.1902 - val_acc: 0.9210\n",
      "Epoch 9/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1859 - acc: 0.9225 - val_loss: 0.1821 - val_acc: 0.9259\n",
      "Epoch 10/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1814 - acc: 0.9246 - val_loss: 0.1829 - val_acc: 0.9281\n",
      "Epoch 11/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1759 - acc: 0.9274 - val_loss: 0.1784 - val_acc: 0.9285\n",
      "Epoch 12/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.1706 - acc: 0.9295 - val_loss: 0.1784 - val_acc: 0.9289\n",
      "Epoch 13/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1666 - acc: 0.9310 - val_loss: 0.1887 - val_acc: 0.9273\n",
      "Epoch 14/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1630 - acc: 0.9330 - val_loss: 0.1880 - val_acc: 0.9282\n",
      "Epoch 15/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1598 - acc: 0.9334 - val_loss: 0.1758 - val_acc: 0.9297\n",
      "Epoch 16/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1561 - acc: 0.9353 - val_loss: 0.1796 - val_acc: 0.9286\n",
      "Epoch 17/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.1536 - acc: 0.9367 - val_loss: 0.1749 - val_acc: 0.9320\n",
      "Epoch 18/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1502 - acc: 0.9384 - val_loss: 0.1765 - val_acc: 0.9323\n",
      "Epoch 19/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.1476 - acc: 0.9391 - val_loss: 0.1793 - val_acc: 0.9318\n",
      "Epoch 20/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1454 - acc: 0.9406 - val_loss: 0.1788 - val_acc: 0.9310\n",
      "Epoch 21/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1418 - acc: 0.9415 - val_loss: 0.1885 - val_acc: 0.9309\n",
      "Epoch 22/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1316 - acc: 0.9459 - val_loss: 0.1707 - val_acc: 0.9331\n",
      "Epoch 23/50\n",
      "228946/228946 [==============================] - 183s 799us/step - loss: 0.1250 - acc: 0.9486 - val_loss: 0.1751 - val_acc: 0.9336\n",
      "Epoch 24/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.1205 - acc: 0.9507 - val_loss: 0.1779 - val_acc: 0.9333\n",
      "Epoch 25/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1183 - acc: 0.9518 - val_loss: 0.1744 - val_acc: 0.9338\n",
      "Epoch 26/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1139 - acc: 0.9538 - val_loss: 0.1792 - val_acc: 0.9333\n",
      "Epoch 27/50\n",
      "228946/228946 [==============================] - 183s 797us/step - loss: 0.1067 - acc: 0.9569 - val_loss: 0.1815 - val_acc: 0.9332\n",
      "Epoch 28/50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1051 - acc: 0.9567 - val_loss: 0.1791 - val_acc: 0.9342\n",
      "Epoch 29/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.1004 - acc: 0.9595 - val_loss: 0.1869 - val_acc: 0.9346\n",
      "Epoch 30/50\n",
      "228946/228946 [==============================] - 183s 798us/step - loss: 0.0987 - acc: 0.9598 - val_loss: 0.1878 - val_acc: 0.9342\n",
      "load model ./log/20180714-022629.Multi_LSTM_CNN_v3.word.022.hdf5\n",
      "Train on 228947 samples, validate on 25439 samples\n",
      "Epoch 1/50\n",
      "228947/228947 [==============================] - 193s 841us/step - loss: 0.4520 - acc: 0.7867 - val_loss: 0.3331 - val_acc: 0.8627\n",
      "Epoch 2/50\n",
      "228947/228947 [==============================] - 183s 798us/step - loss: 0.3119 - acc: 0.8625 - val_loss: 0.2701 - val_acc: 0.8844\n",
      "Epoch 3/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.2677 - acc: 0.8841 - val_loss: 0.2620 - val_acc: 0.8897\n",
      "Epoch 4/50\n",
      "228947/228947 [==============================] - 183s 801us/step - loss: 0.2422 - acc: 0.8963 - val_loss: 0.2239 - val_acc: 0.9073\n",
      "Epoch 5/50\n",
      "228947/228947 [==============================] - 183s 797us/step - loss: 0.2240 - acc: 0.9057 - val_loss: 0.2114 - val_acc: 0.9143\n",
      "Epoch 6/50\n",
      "228947/228947 [==============================] - 183s 797us/step - loss: 0.2110 - acc: 0.9112 - val_loss: 0.2037 - val_acc: 0.9157\n",
      "Epoch 7/50\n",
      "228947/228947 [==============================] - 183s 798us/step - loss: 0.1998 - acc: 0.9165 - val_loss: 0.1983 - val_acc: 0.9201\n",
      "Epoch 8/50\n",
      "228947/228947 [==============================] - 183s 800us/step - loss: 0.1927 - acc: 0.9195 - val_loss: 0.1914 - val_acc: 0.9231\n",
      "Epoch 9/50\n",
      "228947/228947 [==============================] - 183s 798us/step - loss: 0.1855 - acc: 0.9231 - val_loss: 0.1943 - val_acc: 0.9231\n",
      "Epoch 10/50\n",
      "228947/228947 [==============================] - 183s 798us/step - loss: 0.1779 - acc: 0.9257 - val_loss: 0.1913 - val_acc: 0.9235\n",
      "Epoch 11/50\n",
      "228947/228947 [==============================] - 183s 800us/step - loss: 0.1731 - acc: 0.9283 - val_loss: 0.1899 - val_acc: 0.9252\n",
      "Epoch 12/50\n",
      "228947/228947 [==============================] - 183s 798us/step - loss: 0.1686 - acc: 0.9300 - val_loss: 0.1840 - val_acc: 0.9271\n",
      "Epoch 13/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1646 - acc: 0.9323 - val_loss: 0.1843 - val_acc: 0.9277\n",
      "Epoch 14/50\n",
      "228947/228947 [==============================] - 193s 843us/step - loss: 0.1614 - acc: 0.9329 - val_loss: 0.1883 - val_acc: 0.9281\n",
      "Epoch 15/50\n",
      "228947/228947 [==============================] - 194s 849us/step - loss: 0.1590 - acc: 0.9344 - val_loss: 0.1838 - val_acc: 0.9284\n",
      "Epoch 16/50\n",
      "228947/228947 [==============================] - 194s 846us/step - loss: 0.1549 - acc: 0.9362 - val_loss: 0.1861 - val_acc: 0.9286\n",
      "Epoch 17/50\n",
      "228947/228947 [==============================] - 193s 843us/step - loss: 0.1516 - acc: 0.9372 - val_loss: 0.1837 - val_acc: 0.9288\n",
      "Epoch 18/50\n",
      "228947/228947 [==============================] - 184s 803us/step - loss: 0.1499 - acc: 0.9381 - val_loss: 0.1780 - val_acc: 0.9292\n",
      "Epoch 19/50\n",
      "228947/228947 [==============================] - 183s 797us/step - loss: 0.1456 - acc: 0.9402 - val_loss: 0.1853 - val_acc: 0.9289\n",
      "Epoch 20/50\n",
      "228947/228947 [==============================] - 183s 797us/step - loss: 0.1444 - acc: 0.9402 - val_loss: 0.1952 - val_acc: 0.9270\n",
      "Epoch 21/50\n",
      "228947/228947 [==============================] - 183s 798us/step - loss: 0.1432 - acc: 0.9411 - val_loss: 0.1785 - val_acc: 0.9301\n",
      "Epoch 22/50\n",
      "228947/228947 [==============================] - 182s 797us/step - loss: 0.1396 - acc: 0.9432 - val_loss: 0.1860 - val_acc: 0.9306\n",
      "Epoch 23/50\n",
      "228947/228947 [==============================] - 183s 797us/step - loss: 0.1291 - acc: 0.9474 - val_loss: 0.1857 - val_acc: 0.9310\n",
      "Epoch 24/50\n",
      "228947/228947 [==============================] - 182s 797us/step - loss: 0.1228 - acc: 0.9497 - val_loss: 0.1863 - val_acc: 0.9314\n",
      "Epoch 25/50\n",
      "228947/228947 [==============================] - 183s 797us/step - loss: 0.1184 - acc: 0.9516 - val_loss: 0.1882 - val_acc: 0.9308\n",
      "Epoch 26/50\n",
      "228947/228947 [==============================] - 182s 797us/step - loss: 0.1158 - acc: 0.9526 - val_loss: 0.1904 - val_acc: 0.9320\n",
      "load model ./log/20180714-035916.Multi_LSTM_CNN_v3.word.018.hdf5\n",
      "Train on 228947 samples, validate on 25439 samples\n",
      "Epoch 1/50\n",
      "228947/228947 [==============================] - 192s 838us/step - loss: 0.4559 - acc: 0.7848 - val_loss: 0.3246 - val_acc: 0.8612\n",
      "Epoch 2/50\n",
      "228947/228947 [==============================] - 182s 797us/step - loss: 0.3140 - acc: 0.8613 - val_loss: 0.2569 - val_acc: 0.8904\n",
      "Epoch 3/50\n",
      "228947/228947 [==============================] - 186s 810us/step - loss: 0.2701 - acc: 0.8826 - val_loss: 0.2382 - val_acc: 0.9005\n",
      "Epoch 4/50\n",
      "228947/228947 [==============================] - 194s 845us/step - loss: 0.2428 - acc: 0.8961 - val_loss: 0.2176 - val_acc: 0.9097\n",
      "Epoch 5/50\n",
      "228947/228947 [==============================] - 195s 851us/step - loss: 0.2255 - acc: 0.9045 - val_loss: 0.2088 - val_acc: 0.9164\n",
      "Epoch 6/50\n",
      "228947/228947 [==============================] - 194s 845us/step - loss: 0.2115 - acc: 0.9108 - val_loss: 0.2096 - val_acc: 0.9151\n",
      "Epoch 7/50\n",
      "228947/228947 [==============================] - 194s 847us/step - loss: 0.2006 - acc: 0.9156 - val_loss: 0.1935 - val_acc: 0.9226\n",
      "Epoch 8/50\n",
      "228947/228947 [==============================] - 194s 849us/step - loss: 0.1936 - acc: 0.9192 - val_loss: 0.1973 - val_acc: 0.9237\n",
      "Epoch 9/50\n",
      "228947/228947 [==============================] - 194s 847us/step - loss: 0.1867 - acc: 0.9219 - val_loss: 0.1914 - val_acc: 0.9235\n",
      "Epoch 10/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1804 - acc: 0.9250 - val_loss: 0.1848 - val_acc: 0.9268\n",
      "Epoch 11/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1747 - acc: 0.9274 - val_loss: 0.1830 - val_acc: 0.9271\n",
      "Epoch 12/50\n",
      "228947/228947 [==============================] - 183s 800us/step - loss: 0.1705 - acc: 0.9293 - val_loss: 0.1842 - val_acc: 0.9283\n",
      "Epoch 13/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1668 - acc: 0.9308 - val_loss: 0.1800 - val_acc: 0.9293\n",
      "Epoch 14/50\n",
      "228947/228947 [==============================] - 183s 800us/step - loss: 0.1626 - acc: 0.9323 - val_loss: 0.1832 - val_acc: 0.9307\n",
      "Epoch 15/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1587 - acc: 0.9347 - val_loss: 0.1799 - val_acc: 0.9305\n",
      "Epoch 16/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1558 - acc: 0.9354 - val_loss: 0.1779 - val_acc: 0.9323\n",
      "Epoch 17/50\n",
      "228947/228947 [==============================] - 183s 800us/step - loss: 0.1529 - acc: 0.9368 - val_loss: 0.1847 - val_acc: 0.9304\n",
      "Epoch 18/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1501 - acc: 0.9379 - val_loss: 0.1798 - val_acc: 0.9323\n",
      "Epoch 19/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1492 - acc: 0.9385 - val_loss: 0.1815 - val_acc: 0.9316\n",
      "Epoch 20/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1456 - acc: 0.9401 - val_loss: 0.1820 - val_acc: 0.9311\n",
      "Epoch 21/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1335 - acc: 0.9453 - val_loss: 0.1806 - val_acc: 0.9334\n",
      "Epoch 22/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1268 - acc: 0.9477 - val_loss: 0.1834 - val_acc: 0.9345\n",
      "Epoch 23/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1232 - acc: 0.9493 - val_loss: 0.1830 - val_acc: 0.9356\n",
      "Epoch 24/50\n",
      "228947/228947 [==============================] - 183s 799us/step - loss: 0.1195 - acc: 0.9506 - val_loss: 0.1826 - val_acc: 0.9357\n",
      "load model ./log/20180714-052047.Multi_LSTM_CNN_v3.word.016.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/50\n",
      "228948/228948 [==============================] - 194s 849us/step - loss: 0.4526 - acc: 0.7879 - val_loss: 0.3326 - val_acc: 0.8604\n",
      "Epoch 2/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.3100 - acc: 0.8634 - val_loss: 0.2605 - val_acc: 0.8922\n",
      "Epoch 3/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.2665 - acc: 0.8843 - val_loss: 0.2411 - val_acc: 0.9012\n",
      "Epoch 4/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.2420 - acc: 0.8965 - val_loss: 0.2207 - val_acc: 0.9079\n",
      "Epoch 5/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.2236 - acc: 0.9048 - val_loss: 0.2115 - val_acc: 0.9142\n",
      "Epoch 6/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.2103 - acc: 0.9106 - val_loss: 0.2112 - val_acc: 0.9125\n",
      "Epoch 7/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.2002 - acc: 0.9160 - val_loss: 0.1928 - val_acc: 0.9222\n",
      "Epoch 8/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1920 - acc: 0.9195 - val_loss: 0.1941 - val_acc: 0.9234\n",
      "Epoch 9/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1850 - acc: 0.9224 - val_loss: 0.1983 - val_acc: 0.9214\n",
      "Epoch 10/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1784 - acc: 0.9259 - val_loss: 0.1912 - val_acc: 0.9252\n",
      "Epoch 11/50\n",
      "228948/228948 [==============================] - 185s 806us/step - loss: 0.1735 - acc: 0.9277 - val_loss: 0.1918 - val_acc: 0.9249\n",
      "Epoch 12/50\n",
      "228948/228948 [==============================] - 189s 825us/step - loss: 0.1698 - acc: 0.9293 - val_loss: 0.1818 - val_acc: 0.9281\n",
      "Epoch 13/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1646 - acc: 0.9324 - val_loss: 0.1823 - val_acc: 0.9274\n",
      "Epoch 14/50\n",
      "228948/228948 [==============================] - 184s 805us/step - loss: 0.1610 - acc: 0.9337 - val_loss: 0.1863 - val_acc: 0.9299\n",
      "Epoch 15/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1586 - acc: 0.9349 - val_loss: 0.1845 - val_acc: 0.9299\n",
      "Epoch 16/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1547 - acc: 0.9362 - val_loss: 0.1766 - val_acc: 0.9304\n",
      "Epoch 17/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1525 - acc: 0.9373 - val_loss: 0.1853 - val_acc: 0.9291\n",
      "Epoch 18/50\n",
      "228948/228948 [==============================] - 184s 803us/step - loss: 0.1498 - acc: 0.9380 - val_loss: 0.1892 - val_acc: 0.9281\n",
      "Epoch 19/50\n",
      "228948/228948 [==============================] - 184s 804us/step - loss: 0.1467 - acc: 0.9397 - val_loss: 0.1853 - val_acc: 0.9305\n",
      "Epoch 20/50\n",
      "228948/228948 [==============================] - 188s 820us/step - loss: 0.1434 - acc: 0.9410 - val_loss: 0.1790 - val_acc: 0.9323\n",
      "Epoch 21/50\n",
      "228948/228948 [==============================] - 184s 803us/step - loss: 0.1333 - acc: 0.9451 - val_loss: 0.1853 - val_acc: 0.9301\n",
      "Epoch 22/50\n",
      "228948/228948 [==============================] - 187s 817us/step - loss: 0.1259 - acc: 0.9483 - val_loss: 0.1827 - val_acc: 0.9329\n",
      "Epoch 23/50\n",
      "228948/228948 [==============================] - 188s 819us/step - loss: 0.1221 - acc: 0.9498 - val_loss: 0.1899 - val_acc: 0.9316\n",
      "Epoch 24/50\n",
      "228948/228948 [==============================] - 187s 819us/step - loss: 0.1196 - acc: 0.9515 - val_loss: 0.1905 - val_acc: 0.9310\n",
      "load model ./log/20180714-063650.Multi_LSTM_CNN_v3.word.016.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/50\n",
      "228948/228948 [==============================] - 195s 852us/step - loss: 0.4557 - acc: 0.7862 - val_loss: 0.3668 - val_acc: 0.8554\n",
      "Epoch 2/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.3118 - acc: 0.8624 - val_loss: 0.2638 - val_acc: 0.8891\n",
      "Epoch 3/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.2663 - acc: 0.8851 - val_loss: 0.2379 - val_acc: 0.8997\n",
      "Epoch 4/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.2416 - acc: 0.8963 - val_loss: 0.2305 - val_acc: 0.9072\n",
      "Epoch 5/50\n",
      "228948/228948 [==============================] - 185s 810us/step - loss: 0.2243 - acc: 0.9055 - val_loss: 0.2165 - val_acc: 0.9110\n",
      "Epoch 6/50\n",
      "228948/228948 [==============================] - 186s 813us/step - loss: 0.2114 - acc: 0.9111 - val_loss: 0.2088 - val_acc: 0.9156\n",
      "Epoch 7/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.2008 - acc: 0.9160 - val_loss: 0.2005 - val_acc: 0.9181\n",
      "Epoch 8/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1923 - acc: 0.9195 - val_loss: 0.1972 - val_acc: 0.9210\n",
      "Epoch 9/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.1864 - acc: 0.9225 - val_loss: 0.1995 - val_acc: 0.9202\n",
      "Epoch 10/50\n",
      "228948/228948 [==============================] - 184s 804us/step - loss: 0.1792 - acc: 0.9256 - val_loss: 0.1909 - val_acc: 0.9234\n",
      "Epoch 11/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1732 - acc: 0.9279 - val_loss: 0.1877 - val_acc: 0.9245\n",
      "Epoch 12/50\n",
      "228948/228948 [==============================] - 185s 810us/step - loss: 0.1690 - acc: 0.9304 - val_loss: 0.1970 - val_acc: 0.9250\n",
      "Epoch 13/50\n",
      "228948/228948 [==============================] - 188s 819us/step - loss: 0.1656 - acc: 0.9317 - val_loss: 0.1894 - val_acc: 0.9258\n",
      "Epoch 14/50\n",
      "228948/228948 [==============================] - 189s 823us/step - loss: 0.1620 - acc: 0.9334 - val_loss: 0.1872 - val_acc: 0.9260\n",
      "Epoch 15/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1580 - acc: 0.9352 - val_loss: 0.1876 - val_acc: 0.9280\n",
      "Epoch 16/50\n",
      "228948/228948 [==============================] - 184s 803us/step - loss: 0.1555 - acc: 0.9365 - val_loss: 0.1888 - val_acc: 0.9260\n",
      "Epoch 17/50\n",
      "228948/228948 [==============================] - 184s 804us/step - loss: 0.1522 - acc: 0.9372 - val_loss: 0.1903 - val_acc: 0.9278\n",
      "Epoch 18/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1494 - acc: 0.9381 - val_loss: 0.1921 - val_acc: 0.9260\n",
      "Epoch 19/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1384 - acc: 0.9432 - val_loss: 0.1828 - val_acc: 0.9302\n",
      "Epoch 20/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1318 - acc: 0.9464 - val_loss: 0.1851 - val_acc: 0.9307\n",
      "Epoch 21/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.1280 - acc: 0.9473 - val_loss: 0.1879 - val_acc: 0.9289\n",
      "Epoch 22/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1250 - acc: 0.9486 - val_loss: 0.1888 - val_acc: 0.9305\n",
      "Epoch 23/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1218 - acc: 0.9506 - val_loss: 0.1908 - val_acc: 0.9297\n",
      "Epoch 24/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.1145 - acc: 0.9529 - val_loss: 0.1881 - val_acc: 0.9312\n",
      "Epoch 25/50\n",
      "228948/228948 [==============================] - 187s 818us/step - loss: 0.1112 - acc: 0.9544 - val_loss: 0.1901 - val_acc: 0.9318\n",
      "Epoch 26/50\n",
      "228948/228948 [==============================] - 188s 820us/step - loss: 0.1070 - acc: 0.9562 - val_loss: 0.1949 - val_acc: 0.9305\n",
      "Epoch 27/50\n",
      "228948/228948 [==============================] - 186s 814us/step - loss: 0.1058 - acc: 0.9568 - val_loss: 0.1962 - val_acc: 0.9307\n",
      "load model ./log/20180714-075224.Multi_LSTM_CNN_v3.word.019.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/50\n",
      "228948/228948 [==============================] - 201s 877us/step - loss: 0.4605 - acc: 0.7839 - val_loss: 0.3465 - val_acc: 0.8597\n",
      "Epoch 2/50\n",
      "228948/228948 [==============================] - 190s 832us/step - loss: 0.3171 - acc: 0.8584 - val_loss: 0.2728 - val_acc: 0.8883\n",
      "Epoch 3/50\n",
      "228948/228948 [==============================] - 189s 827us/step - loss: 0.2705 - acc: 0.8828 - val_loss: 0.2458 - val_acc: 0.9003\n",
      "Epoch 4/50\n",
      "228948/228948 [==============================] - 193s 842us/step - loss: 0.2434 - acc: 0.8961 - val_loss: 0.2130 - val_acc: 0.9104\n",
      "Epoch 5/50\n",
      "228948/228948 [==============================] - 193s 842us/step - loss: 0.2266 - acc: 0.9038 - val_loss: 0.2144 - val_acc: 0.9143\n",
      "Epoch 6/50\n",
      "228948/228948 [==============================] - 193s 841us/step - loss: 0.2113 - acc: 0.9105 - val_loss: 0.2049 - val_acc: 0.9180\n",
      "Epoch 7/50\n",
      "228948/228948 [==============================] - 194s 845us/step - loss: 0.2016 - acc: 0.9157 - val_loss: 0.1934 - val_acc: 0.9194\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/50\n",
      "228948/228948 [==============================] - 193s 845us/step - loss: 0.1937 - acc: 0.9196 - val_loss: 0.1876 - val_acc: 0.9241\n",
      "Epoch 9/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.1852 - acc: 0.9223 - val_loss: 0.1893 - val_acc: 0.9248\n",
      "Epoch 10/50\n",
      "228948/228948 [==============================] - 187s 816us/step - loss: 0.1797 - acc: 0.9254 - val_loss: 0.1880 - val_acc: 0.9253\n",
      "Epoch 11/50\n",
      "228948/228948 [==============================] - 190s 830us/step - loss: 0.1753 - acc: 0.9268 - val_loss: 0.1789 - val_acc: 0.9292\n",
      "Epoch 12/50\n",
      "228948/228948 [==============================] - 188s 822us/step - loss: 0.1714 - acc: 0.9289 - val_loss: 0.1900 - val_acc: 0.9274\n",
      "Epoch 13/50\n",
      "228948/228948 [==============================] - 191s 836us/step - loss: 0.1657 - acc: 0.9312 - val_loss: 0.1814 - val_acc: 0.9314\n",
      "Epoch 14/50\n",
      "228948/228948 [==============================] - 193s 842us/step - loss: 0.1616 - acc: 0.9330 - val_loss: 0.1865 - val_acc: 0.9305\n",
      "Epoch 15/50\n",
      "228948/228948 [==============================] - 192s 841us/step - loss: 0.1586 - acc: 0.9341 - val_loss: 0.1813 - val_acc: 0.9309\n",
      "Epoch 16/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.1465 - acc: 0.9396 - val_loss: 0.1771 - val_acc: 0.9318\n",
      "Epoch 17/50\n",
      "228948/228948 [==============================] - 191s 835us/step - loss: 0.1384 - acc: 0.9430 - val_loss: 0.1811 - val_acc: 0.9304\n",
      "Epoch 18/50\n",
      "228948/228948 [==============================] - 192s 840us/step - loss: 0.1356 - acc: 0.9440 - val_loss: 0.1738 - val_acc: 0.9332\n",
      "Epoch 19/50\n",
      "228948/228948 [==============================] - 193s 844us/step - loss: 0.1317 - acc: 0.9456 - val_loss: 0.1756 - val_acc: 0.9334\n",
      "Epoch 20/50\n",
      "228948/228948 [==============================] - 192s 839us/step - loss: 0.1284 - acc: 0.9473 - val_loss: 0.1809 - val_acc: 0.9333\n",
      "Epoch 21/50\n",
      "228948/228948 [==============================] - 191s 833us/step - loss: 0.1260 - acc: 0.9484 - val_loss: 0.1789 - val_acc: 0.9331\n",
      "Epoch 22/50\n",
      "228948/228948 [==============================] - 193s 844us/step - loss: 0.1223 - acc: 0.9493 - val_loss: 0.1870 - val_acc: 0.9338\n",
      "Epoch 23/50\n",
      "228948/228948 [==============================] - 189s 826us/step - loss: 0.1154 - acc: 0.9525 - val_loss: 0.1818 - val_acc: 0.9342\n",
      "Epoch 24/50\n",
      "228948/228948 [==============================] - 189s 825us/step - loss: 0.1105 - acc: 0.9550 - val_loss: 0.1855 - val_acc: 0.9346\n",
      "Epoch 25/50\n",
      "228948/228948 [==============================] - 192s 839us/step - loss: 0.1069 - acc: 0.9563 - val_loss: 0.1883 - val_acc: 0.9352\n",
      "Epoch 26/50\n",
      "228948/228948 [==============================] - 192s 840us/step - loss: 0.1051 - acc: 0.9567 - val_loss: 0.1866 - val_acc: 0.9362\n",
      "load model ./log/20180714-091725.Multi_LSTM_CNN_v3.word.018.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/50\n",
      "228948/228948 [==============================] - 197s 860us/step - loss: 0.4545 - acc: 0.7857 - val_loss: 0.3420 - val_acc: 0.8628\n",
      "Epoch 2/50\n",
      "228948/228948 [==============================] - 188s 821us/step - loss: 0.3120 - acc: 0.8619 - val_loss: 0.2685 - val_acc: 0.8865\n",
      "Epoch 3/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.2683 - acc: 0.8838 - val_loss: 0.2295 - val_acc: 0.9047\n",
      "Epoch 4/50\n",
      "228948/228948 [==============================] - 190s 828us/step - loss: 0.2424 - acc: 0.8961 - val_loss: 0.2594 - val_acc: 0.8990\n",
      "Epoch 5/50\n",
      "228948/228948 [==============================] - 192s 840us/step - loss: 0.2245 - acc: 0.9040 - val_loss: 0.2110 - val_acc: 0.9134\n",
      "Epoch 6/50\n",
      "228948/228948 [==============================] - 191s 835us/step - loss: 0.2114 - acc: 0.9107 - val_loss: 0.1995 - val_acc: 0.9191\n",
      "Epoch 7/50\n",
      "228948/228948 [==============================] - 194s 848us/step - loss: 0.2013 - acc: 0.9152 - val_loss: 0.2047 - val_acc: 0.9202\n",
      "Epoch 8/50\n",
      "228948/228948 [==============================] - 194s 847us/step - loss: 0.1929 - acc: 0.9191 - val_loss: 0.2012 - val_acc: 0.9190\n",
      "Epoch 9/50\n",
      "228948/228948 [==============================] - 192s 840us/step - loss: 0.1859 - acc: 0.9226 - val_loss: 0.1924 - val_acc: 0.9245\n",
      "Epoch 10/50\n",
      "228948/228948 [==============================] - 194s 847us/step - loss: 0.1786 - acc: 0.9259 - val_loss: 0.1902 - val_acc: 0.9251\n",
      "Epoch 11/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1740 - acc: 0.9279 - val_loss: 0.1978 - val_acc: 0.9246\n",
      "Epoch 12/50\n",
      "228948/228948 [==============================] - 183s 797us/step - loss: 0.1700 - acc: 0.9296 - val_loss: 0.1838 - val_acc: 0.9278\n",
      "Epoch 13/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1652 - acc: 0.9314 - val_loss: 0.1848 - val_acc: 0.9296\n",
      "Epoch 14/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1627 - acc: 0.9329 - val_loss: 0.1837 - val_acc: 0.9288\n",
      "Epoch 15/50\n",
      "228948/228948 [==============================] - 183s 797us/step - loss: 0.1591 - acc: 0.9346 - val_loss: 0.1796 - val_acc: 0.9320\n",
      "Epoch 16/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1549 - acc: 0.9360 - val_loss: 0.1864 - val_acc: 0.9294\n",
      "Epoch 17/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1527 - acc: 0.9369 - val_loss: 0.1854 - val_acc: 0.9296\n",
      "Epoch 18/50\n",
      "228948/228948 [==============================] - 185s 807us/step - loss: 0.1479 - acc: 0.9387 - val_loss: 0.1807 - val_acc: 0.9321\n",
      "Epoch 19/50\n",
      "228948/228948 [==============================] - 194s 848us/step - loss: 0.1457 - acc: 0.9398 - val_loss: 0.1824 - val_acc: 0.9301\n",
      "Epoch 20/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1361 - acc: 0.9431 - val_loss: 0.1816 - val_acc: 0.9325\n",
      "Epoch 21/50\n",
      "228948/228948 [==============================] - 193s 844us/step - loss: 0.1289 - acc: 0.9473 - val_loss: 0.1784 - val_acc: 0.9336\n",
      "Epoch 22/50\n",
      "228948/228948 [==============================] - 192s 841us/step - loss: 0.1237 - acc: 0.9489 - val_loss: 0.1823 - val_acc: 0.9338\n",
      "Epoch 23/50\n",
      "228948/228948 [==============================] - 194s 847us/step - loss: 0.1211 - acc: 0.9499 - val_loss: 0.1829 - val_acc: 0.9337\n",
      "Epoch 24/50\n",
      "228948/228948 [==============================] - 194s 849us/step - loss: 0.1182 - acc: 0.9517 - val_loss: 0.1879 - val_acc: 0.9346\n",
      "Epoch 25/50\n",
      "228948/228948 [==============================] - 195s 852us/step - loss: 0.1142 - acc: 0.9527 - val_loss: 0.1866 - val_acc: 0.9344\n",
      "Epoch 26/50\n",
      "228948/228948 [==============================] - 193s 844us/step - loss: 0.1085 - acc: 0.9555 - val_loss: 0.1898 - val_acc: 0.9348\n",
      "Epoch 27/50\n",
      "228948/228948 [==============================] - 195s 854us/step - loss: 0.1043 - acc: 0.9574 - val_loss: 0.1972 - val_acc: 0.9341\n",
      "Epoch 28/50\n",
      "228948/228948 [==============================] - 194s 847us/step - loss: 0.1016 - acc: 0.9583 - val_loss: 0.1924 - val_acc: 0.9351\n",
      "Epoch 29/50\n",
      "228948/228948 [==============================] - 195s 852us/step - loss: 0.0987 - acc: 0.9595 - val_loss: 0.1995 - val_acc: 0.9334\n",
      "load model ./log/20180714-104217.Multi_LSTM_CNN_v3.word.021.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/50\n",
      "228948/228948 [==============================] - 197s 862us/step - loss: 0.4546 - acc: 0.7866 - val_loss: 0.3706 - val_acc: 0.8559\n",
      "Epoch 2/50\n",
      "228948/228948 [==============================] - 183s 797us/step - loss: 0.3124 - acc: 0.8618 - val_loss: 0.2797 - val_acc: 0.8825\n",
      "Epoch 3/50\n",
      "228948/228948 [==============================] - 182s 797us/step - loss: 0.2691 - acc: 0.8833 - val_loss: 0.2529 - val_acc: 0.8946\n",
      "Epoch 4/50\n",
      "228948/228948 [==============================] - 183s 797us/step - loss: 0.2427 - acc: 0.8959 - val_loss: 0.2257 - val_acc: 0.9073\n",
      "Epoch 5/50\n",
      "228948/228948 [==============================] - 186s 810us/step - loss: 0.2239 - acc: 0.9048 - val_loss: 0.2146 - val_acc: 0.9137\n",
      "Epoch 6/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.2115 - acc: 0.9107 - val_loss: 0.2150 - val_acc: 0.9150\n",
      "Epoch 7/50\n",
      "228948/228948 [==============================] - 182s 797us/step - loss: 0.2017 - acc: 0.9161 - val_loss: 0.2074 - val_acc: 0.9158\n",
      "Epoch 8/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1926 - acc: 0.9195 - val_loss: 0.2040 - val_acc: 0.9183\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/50\n",
      "228948/228948 [==============================] - 182s 797us/step - loss: 0.1863 - acc: 0.9223 - val_loss: 0.1923 - val_acc: 0.9210\n",
      "Epoch 10/50\n",
      "228948/228948 [==============================] - 183s 797us/step - loss: 0.1792 - acc: 0.9252 - val_loss: 0.1962 - val_acc: 0.9232\n",
      "Epoch 11/50\n",
      "228948/228948 [==============================] - 183s 798us/step - loss: 0.1748 - acc: 0.9274 - val_loss: 0.1931 - val_acc: 0.9239\n",
      "Epoch 12/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1697 - acc: 0.9298 - val_loss: 0.1927 - val_acc: 0.9253\n",
      "Epoch 13/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1654 - acc: 0.9320 - val_loss: 0.1829 - val_acc: 0.9264\n",
      "Epoch 14/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1620 - acc: 0.9329 - val_loss: 0.1879 - val_acc: 0.9274\n",
      "Epoch 15/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.1585 - acc: 0.9344 - val_loss: 0.1861 - val_acc: 0.9287\n",
      "Epoch 16/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1548 - acc: 0.9366 - val_loss: 0.1821 - val_acc: 0.9291\n",
      "Epoch 17/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1508 - acc: 0.9375 - val_loss: 0.1825 - val_acc: 0.9282\n",
      "Epoch 18/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1496 - acc: 0.9384 - val_loss: 0.1899 - val_acc: 0.9276\n",
      "Epoch 19/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1468 - acc: 0.9397 - val_loss: 0.1892 - val_acc: 0.9288\n",
      "Epoch 20/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1447 - acc: 0.9405 - val_loss: 0.2003 - val_acc: 0.9259\n",
      "Epoch 21/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.1327 - acc: 0.9455 - val_loss: 0.1838 - val_acc: 0.9318\n",
      "Epoch 22/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1267 - acc: 0.9483 - val_loss: 0.1858 - val_acc: 0.9322\n",
      "Epoch 23/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.1230 - acc: 0.9503 - val_loss: 0.1918 - val_acc: 0.9309\n",
      "Epoch 24/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.1186 - acc: 0.9507 - val_loss: 0.1869 - val_acc: 0.9310\n",
      "load model ./log/20180714-121604.Multi_LSTM_CNN_v3.word.016.hdf5\n",
      "Train on 228948 samples, validate on 25438 samples\n",
      "Epoch 1/50\n",
      "228948/228948 [==============================] - 203s 888us/step - loss: 0.4625 - acc: 0.7815 - val_loss: 0.3640 - val_acc: 0.8582\n",
      "Epoch 2/50\n",
      "228948/228948 [==============================] - 191s 835us/step - loss: 0.3161 - acc: 0.8602 - val_loss: 0.2667 - val_acc: 0.8884\n",
      "Epoch 3/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.2700 - acc: 0.8831 - val_loss: 0.2396 - val_acc: 0.8996\n",
      "Epoch 4/50\n",
      "228948/228948 [==============================] - 189s 825us/step - loss: 0.2433 - acc: 0.8958 - val_loss: 0.2214 - val_acc: 0.9092\n",
      "Epoch 5/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.2255 - acc: 0.9043 - val_loss: 0.2047 - val_acc: 0.9133\n",
      "Epoch 6/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.2110 - acc: 0.9105 - val_loss: 0.1970 - val_acc: 0.9186\n",
      "Epoch 7/50\n",
      "228948/228948 [==============================] - 186s 813us/step - loss: 0.2004 - acc: 0.9157 - val_loss: 0.1925 - val_acc: 0.9212\n",
      "Epoch 8/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1938 - acc: 0.9198 - val_loss: 0.1983 - val_acc: 0.9204\n",
      "Epoch 9/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1882 - acc: 0.9213 - val_loss: 0.1863 - val_acc: 0.9246\n",
      "Epoch 10/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1796 - acc: 0.9250 - val_loss: 0.1911 - val_acc: 0.9248\n",
      "Epoch 11/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1750 - acc: 0.9276 - val_loss: 0.1856 - val_acc: 0.9281\n",
      "Epoch 12/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.1693 - acc: 0.9300 - val_loss: 0.1842 - val_acc: 0.9293\n",
      "Epoch 13/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1668 - acc: 0.9303 - val_loss: 0.1861 - val_acc: 0.9285\n",
      "Epoch 14/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1627 - acc: 0.9327 - val_loss: 0.1785 - val_acc: 0.9305\n",
      "Epoch 15/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1582 - acc: 0.9355 - val_loss: 0.1779 - val_acc: 0.9285\n",
      "Epoch 16/50\n",
      "228948/228948 [==============================] - 183s 799us/step - loss: 0.1555 - acc: 0.9354 - val_loss: 0.1765 - val_acc: 0.9316\n",
      "Epoch 17/50\n",
      "228948/228948 [==============================] - 183s 800us/step - loss: 0.1530 - acc: 0.9369 - val_loss: 0.1824 - val_acc: 0.9323\n",
      "Epoch 18/50\n",
      "228948/228948 [==============================] - 184s 804us/step - loss: 0.1499 - acc: 0.9384 - val_loss: 0.1744 - val_acc: 0.9312\n",
      "Epoch 19/50\n",
      "228948/228948 [==============================] - 187s 816us/step - loss: 0.1470 - acc: 0.9391 - val_loss: 0.1770 - val_acc: 0.9310\n",
      "Epoch 20/50\n",
      "228948/228948 [==============================] - 189s 824us/step - loss: 0.1453 - acc: 0.9400 - val_loss: 0.1841 - val_acc: 0.9308\n",
      "Epoch 21/50\n",
      "228948/228948 [==============================] - 193s 841us/step - loss: 0.1434 - acc: 0.9409 - val_loss: 0.1813 - val_acc: 0.9310\n",
      "Epoch 22/50\n",
      "228948/228948 [==============================] - 193s 844us/step - loss: 0.1393 - acc: 0.9426 - val_loss: 0.1813 - val_acc: 0.9314\n",
      "Epoch 23/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.1289 - acc: 0.9467 - val_loss: 0.1790 - val_acc: 0.9332\n",
      "Epoch 24/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.1232 - acc: 0.9497 - val_loss: 0.1788 - val_acc: 0.9325\n",
      "Epoch 25/50\n",
      "228948/228948 [==============================] - 183s 801us/step - loss: 0.1190 - acc: 0.9510 - val_loss: 0.1861 - val_acc: 0.9314\n",
      "Epoch 26/50\n",
      "228948/228948 [==============================] - 184s 802us/step - loss: 0.1150 - acc: 0.9530 - val_loss: 0.1825 - val_acc: 0.9333\n",
      "load model ./log/20180714-133147.Multi_LSTM_CNN_v3.word.018.hdf5\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "best_results = []\n",
    "# last_results = []\n",
    "best_file_names = []\n",
    "dev_predictions = []\n",
    "\n",
    "for i, (train_index, dev_index) in enumerate(StratifiedKFold(n_splits=10).split(X=word1, y=label)):  # word/char switch\n",
    "    print(\"-\" * 60)\n",
    "    print(\"Fold {} training start...\".format(i))\n",
    "    \n",
    "    train_x1, train_x2, train_y = word1[train_index, :], word2[train_index, :], label[train_index]  # word/char switch\n",
    "    dev_x1, dev_x2, dev_y = word1[dev_index, :], word2[dev_index, :], label[dev_index]  # word/char switch\n",
    "    \n",
    "    input1 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "    input2 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "    embedding_layer = Embedding(\n",
    "        input_dim=word_embedding_data.shape[0],  # word/char switch\n",
    "        output_dim=word_embedding_data.shape[1],  # word/char switch\n",
    "        weights=[word_embedding_data],  # word/char switch\n",
    "        input_length=SEQ_LEN,\n",
    "        trainable=False\n",
    "    )\n",
    "    \n",
    "    vector1 = embedding_layer(input1)\n",
    "    vector2 = embedding_layer(input2)\n",
    "    \n",
    "    lstm_layer1 = LSTM(LSTM_SIZE1, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "    layer1a = lstm_layer1(vector1)\n",
    "    layer1a = Dropout(LSTM_DROP_RATE)(layer1a)\n",
    "    layer1b = lstm_layer1(vector2)\n",
    "    layer1b = Dropout(LSTM_DROP_RATE)(layer1b)\n",
    "\n",
    "    lstm_layer2 = LSTM(LSTM_SIZE2, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "    layer2a = lstm_layer2(layer1a)\n",
    "    layer2b = lstm_layer2(layer1b)\n",
    "    \n",
    "    conv1a, conv1b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN1, kernel_size=1)\n",
    "    conv2a, conv2b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN2, kernel_size=2)\n",
    "    conv3a, conv3b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN3, kernel_size=3)\n",
    "    conv4a, conv4b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN4, kernel_size=4)\n",
    "    conv5a, conv5b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN5, kernel_size=5)\n",
    "    conv6a, conv6b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN6, kernel_size=6)\n",
    "    \n",
    "    merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "    merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "    diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "    mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "    merge = concatenate([diff, mult])\n",
    "    \n",
    "    x = Dropout(DROP_RATE)(merge)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(DENSE_SIZE1, activation=\"relu\")(x)\n",
    "    x = Dropout(DROP_RATE)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(DENSE_SIZE2, activation=\"relu\")(x)\n",
    "    x = Dropout(DROP_RATE)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "    \n",
    "    model = Model(inputs=[input1, input2], outputs=pred)\n",
    "    model.compile(\n",
    "        optimizer=\"nadam\",\n",
    "        loss=\"binary_crossentropy\",\n",
    "        metrics=[\"acc\"]\n",
    "    )\n",
    "    \n",
    "    early_stopping = EarlyStopping(\"val_loss\", patience=8)\n",
    "    lr_reducer = ReduceLROnPlateau(factor=0.5, patience=4, min_lr=0.0005)\n",
    "    check_point = ModelCheckpoint(\n",
    "        \"./log/%s.Multi_LSTM_CNN_v3.word.{epoch:03d}.hdf5\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "        monitor=\"val_loss\",\n",
    "        save_best_only=True,\n",
    "    )\n",
    "    \n",
    "    fit_res = model.fit(\n",
    "        x=[train_x1, train_x2],\n",
    "        y=train_y,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=NUM_EPOCHES,\n",
    "        validation_data=([dev_x1, dev_x2], dev_y),\n",
    "        shuffle=True,\n",
    "        callbacks=[early_stopping, lr_reducer, check_point]\n",
    "    )\n",
    "    \n",
    "#     pred_last = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "#     last_results.append(pd.DataFrame(pred_last, columns=[\"y_pre\"]))\n",
    "    \n",
    "    best_model_file = glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\")\n",
    "    best_file_names.append(best_model_file)\n",
    "    print(\"load model %s\" % (best_model_file,))\n",
    "    model.load_weights(best_model_file)\n",
    "    pred_best = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "    best_results.append(pd.DataFrame(pred_best, columns=[\"y_pre\"]))\n",
    "    \n",
    "    dev_pred = model.predict([dev_x1, dev_x2], batch_size=BATCH_SIZE)\n",
    "    dev_result = pd.DataFrame({\"pred\": dev_pred.ravel(), \"label\": dev_y})\n",
    "    dev_predictions.append(dev_result)\n",
    "\n",
    "# pd.DataFrame(pd.concat(last_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "#     \"./result/%s-Multi_LSTM_CNN_v3_word_last.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "#     index=False\n",
    "# )\n",
    "pd.DataFrame(pd.concat(best_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "    \"./result/%s-Multi_LSTM_CNN_v3_word_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "    index=False\n",
    ")\n",
    "\n",
    "total_dev = pd.concat(dev_predictions, axis=0)\n",
    "total_dev.to_csv(\n",
    "    \"./result/%s-Multi_LSTM_CNN_v3_word_dev_result.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
    "    index=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# CHARS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# best_results = []\n",
    "# # last_results = []\n",
    "# best_file_names = []\n",
    "# dev_predictions = []\n",
    "\n",
    "# for i, (train_index, dev_index) in enumerate(StratifiedKFold(n_splits=10).split(X=char1, y=label)):  # word/char switch\n",
    "#     print(\"-\" * 60)\n",
    "#     print(\"Fold {} training start...\".format(i))\n",
    "    \n",
    "#     train_x1, train_x2, train_y = char1[train_index, :], char2[train_index, :], label[train_index]  # word/char switch\n",
    "#     dev_x1, dev_x2, dev_y = char1[dev_index, :], char2[dev_index, :], label[dev_index]  # word/char switch\n",
    "    \n",
    "#     input1 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "#     input2 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "#     embedding_layer = Embedding(\n",
    "#         input_dim=char_embedding_data.shape[0],  # word/char switch\n",
    "#         output_dim=char_embedding_data.shape[1],  # word/char switch\n",
    "#         weights=[char_embedding_data],  # word/char switch\n",
    "#         input_length=SEQ_LEN,\n",
    "#         trainable=False\n",
    "#     )\n",
    "    \n",
    "#     vector1 = embedding_layer(input1)\n",
    "#     vector2 = embedding_layer(input2)\n",
    "    \n",
    "#     lstm_layer1 = LSTM(LSTM_SIZE1, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "#     layer1a = lstm_layer1(vector1)\n",
    "#     layer1a = Dropout(LSTM_DROP_RATE)(layer1a)\n",
    "#     layer1b = lstm_layer1(vector2)\n",
    "#     layer1b = Dropout(LSTM_DROP_RATE)(layer1b)\n",
    "\n",
    "#     lstm_layer2 = LSTM(LSTM_SIZE2, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "#     layer2a = lstm_layer2(layer1a)\n",
    "#     layer2b = lstm_layer2(layer1b)\n",
    "    \n",
    "#     conv1a, conv1b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN1, kernel_size=1)\n",
    "#     conv2a, conv2b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN2, kernel_size=2)\n",
    "#     conv3a, conv3b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN3, kernel_size=3)\n",
    "#     conv4a, conv4b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN4, kernel_size=4)\n",
    "#     conv5a, conv5b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN5, kernel_size=5)\n",
    "#     conv6a, conv6b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN6, kernel_size=6)\n",
    "    \n",
    "#     merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "#     merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "#     diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "#     mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "#     merge = concatenate([diff, mult])\n",
    "    \n",
    "#     x = Dropout(DROP_RATE)(merge)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dense(DENSE_SIZE1, activation=\"relu\")(x)\n",
    "#     x = Dropout(DROP_RATE)(x)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dense(DENSE_SIZE2, activation=\"relu\")(x)\n",
    "#     x = Dropout(DROP_RATE)(x)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "    \n",
    "#     model = Model(inputs=[input1, input2], outputs=pred)\n",
    "#     model.compile(\n",
    "#         optimizer=\"nadam\",\n",
    "#         loss=\"binary_crossentropy\",\n",
    "#         metrics=[\"acc\"]\n",
    "#     )\n",
    "    \n",
    "#     early_stopping = EarlyStopping(\"val_loss\", patience=8)\n",
    "#     lr_reducer = ReduceLROnPlateau(factor=0.5, patience=4, min_lr=0.0005)\n",
    "#     check_point = ModelCheckpoint(\n",
    "#         \"./log/%s.Multi_LSTM_CNN_v3.char.{epoch:03d}.hdf5\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "#         monitor=\"val_loss\",\n",
    "#         save_best_only=True,\n",
    "#     )\n",
    "    \n",
    "#     fit_res = model.fit(\n",
    "#         x=[train_x1, train_x2],\n",
    "#         y=train_y,\n",
    "#         batch_size=BATCH_SIZE,\n",
    "#         epochs=NUM_EPOCHES,\n",
    "#         validation_data=([dev_x1, dev_x2], dev_y),\n",
    "#         shuffle=True,\n",
    "#         callbacks=[early_stopping, lr_reducer, check_point]\n",
    "#     )\n",
    "    \n",
    "# #     pred_last = model.predict([test_char1, test_char2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "# #     last_results.append(pd.DataFrame(pred_last, columns=[\"y_pre\"]))\n",
    "    \n",
    "#     best_model_file = glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\")\n",
    "#     best_file_names.append(best_model_file)\n",
    "#     print(\"load model %s\" % (best_model_file,))\n",
    "#     model.load_weights(best_model_file)\n",
    "#     pred_best = model.predict([test_char1, test_char2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "#     best_results.append(pd.DataFrame(pred_best, columns=[\"y_pre\"]))\n",
    "\n",
    "#     dev_pred = model.predict([dev_x1, dev_x2], batch_size=BATCH_SIZE)\n",
    "#     dev_result = pd.DataFrame({\"pred\": dev_pred.ravel(), \"label\": dev_y})\n",
    "#     dev_predictions.append(dev_result)\n",
    "\n",
    "# # pd.DataFrame(pd.concat(last_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "# #     \"./result/%s-Multi_LSTM_CNN_v3_char_last.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "# #     index=False\n",
    "# # )\n",
    "# pd.DataFrame(pd.concat(best_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "#     \"./result/%s-Multi_LSTM_CNN_v3_char_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "#     index=False\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_path = \"./log/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\") + \"/\"\n",
    "os.mkdir(model_path)\n",
    "for model_name in best_file_names:\n",
    "    abs_name = os.path.split(model_name)[1]\n",
    "    os.rename(model_name, model_path + abs_name)\n",
    "    \n",
    "t_label = total_dev[\"label\"].values\n",
    "t_pred = total_dev[\"pred\"].values\n",
    "t_log_loss = -(t_label * np.log(t_pred) + (1 - t_label) * np.log(1 - t_pred))\n",
    "pd.DataFrame(t_log_loss, columns=[\"loss\"]).to_csv(model_path + \"log_loss.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
